package com.catmiao.spark.rdd.operator.transform

import org.apache.spark.{SparkConf, SparkContext}

/**
 * @title: RDD_Operator_01_transform
 * @projectName spark_study
 * @description: TODO
 * @author ChengMiao
 * @date 2024/2/2 11:48
 */
object RDD_Operator_08_transform {

  def main(args: Array[String]): Unit = {
    val sparkCon = new SparkConf().setMaster("local[*]").setAppName("rdd")

    val sparkContext = new SparkContext(sparkCon)


    val rdd = sparkContext.makeRDD(List(1,2,3,5,1,1,2))




    // distinct去重
    /**
     * collection中的distinct是基于hashSet实现；
     * 而rdd中的distinct是基于分布式的方式实现:
     *    case _ => map(x => (x, null)).reduceByKey((x, _) => x, numPartitions).map(_._1)
     *       [1]. (1,null),(2,null)...
     *       [2]. 相同key聚合，只取第一个v  (1,null),(2,null)
     *       [3]. map取得元组的第一个值 1,2,3,5
     *
     */
    //    println(rdd.distinct().collect().mkString(","))
    println(rdd.distinct(2).collect().mkString(","))





    sparkContext.stop()
  }

}
